/*
 * Copyright 2009 BioTeam Inc
 * 
 * Licensed under License GNU LESSER GENERAL PUBLIC LICENSE version 3.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 * 
 * http://www.gnu.org/copyleft/lesser.html
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */
package net.bioteam.appweb.ds;

import java.io.File;
import java.io.FileFilter;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.InputStream;
import java.io.Reader;
import java.io.Writer;
import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.zip.GZIPInputStream;

import org.apache.commons.codec.binary.Hex;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.filefilter.PrefixFileFilter;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

//TODO check diskspace.
//TODO check DB_PATH, DB_TMP(not equal DB_PATH), formatdb/mpiformatb, btformatdb, bpatch binary, btmove


public class DSClient
{
	static Log logger = LogFactory.getLog(DSClient.class);
	//public static String FASTA_SRC_PATH = "d:/tmp";
	public static String DB_PATH = "d:/tmp";
	public static String DB_TMP = "d:/tmp1";
	public static HttpClient client;
	public static Map<String, Subscription> subs;
	public static void main(String args[])
	{
		client = new HttpClient();
		DB_PATH=System.getenv("DB_PATH");
		DB_TMP=System.getenv("DB_TMP");
		DSPersister.SUB_FILE=System.getenv("SUB_FILE");
		try
		{
			File tmpDir=new File(DB_TMP);
			 subs= DSPersister.loadSubscription();
			for (Subscription sub : subs.values())
			{
				logger.info("hanlde dataset --------" + sub.getName());

				if (sub.isLocalds())
				{
					logger.info("\tskip  since it is local dataset");
					continue;
				}
				if (!sub.isEnabled())
				{
					logger.info("\tskip  since it is disabled");
					continue;
				}
				try
				{
					handleOneDataset(sub);
				} catch (Exception e1)
				{
					logger.error("\t"+e1.getMessage(), e1);
				} finally 
				{
					//clean lefted artifact.
					for (File file: tmpDir.listFiles((FileFilter)new PrefixFileFilter(sub.getName())))
					{
						file.delete();
					}
				}
				logger.info("FINISHED dataset --------" + sub.getName());
			}
		} catch (Exception e)
		{
			logger.error(e.getMessage(), e);
		}

	}


	public static void handleOneDataset(Subscription sub) throws Exception
	{
		List<Integer> ids = new ArrayList<Integer>();
		ids.add((int) sub.getServerId());
		List<SrcDataset> dss = DSServer.loadDatasetForDownload(ids);
		if (dss.isEmpty())
		{
			logger.info("\tskip since there is no corresponding server dataset");
			return;
		}
		SrcDataset src = dss.get(0);
		if (!src.isEnabled())
		{
			logger.info("\tskip since the source dataset is disabled");
			return;
		}
		if (src.versions.isEmpty())
		{
			logger.info("\tskip server does not have any data");
			return;
		}

		if (sub.getServerVersion() == null)
		{
			logger.info("\tperform source updating since client does not have source version");
			if (enoughSpace(src.getMostRecentVersion().getFsize()))
				sourceUpdate(sub, src);
			return;
		}
		if (!isDatasetValid(sub))
		{
			logger.info("\t perform source updating since dataset is invalid");
			if (enoughSpace(src.getMostRecentVersion().getFsize()))
				sourceUpdate(sub, src);
			return;
		}

		Version mostRecentVersion = src.getMostRecentVersion();
		if (mostRecentVersion.getVersion().equals(sub.getServerVersion()))
		{
			logger.info("\t skip since no update is available. Most recent version is " + mostRecentVersion.getVersion());
			return;
		}
		Patch patch = src.getPatchByPerversion(sub.getServerVersion());
		if (patch == null)
		{
			// we should have a patch. If we do not, this is a server
			// error, we just start from beginning
			logger.info("\t !!!SHOULD have a patch. but not. Just start from scratch");
			if (enoughSpace(src.getMostRecentVersion().getFsize()))
				sourceUpdate(sub, src);
			return;
		}
		if (src.getVersion(patch.getCurversion())==null)
		{
			logger.info("\t !!!skip since can not find the version corresponding to the patch");
			if (enoughSpace(src.getMostRecentVersion().getFsize()))
				sourceUpdate(sub, src);
			return;
		}
		if (enoughSpace(src.getVersion(patch.getCurversion()).getFsize()))
			patchUpdate(sub, src, patch);
	}
	
	public static boolean enoughSpace(long fsize)
	{
		
		//1.source file, uncompressed file(5 time of source file)=6
		//2. after gunzip, we only has uncompressed file=5.
		long neededSize=fsize*5;
		//#3. after formatt, we increased 1.5 time=6.5
		neededSize*=((double)fsize)*1.5;
		//4. give one 500M extra space for OS and other user.
		neededSize+=500*1024*1024;
		
		if (neededSize>new File(DB_PATH).getUsableSpace())
		{
			logger.info("\t skip since database directory does not have enough space . needed space is "+neededSize);
			return false;
		}
		if (neededSize>new File(DB_TMP).getUsableSpace())
		{
			logger.info("\t skip since tmp directory does not have enough space . needed space is "+neededSize);
			return false;
		}
		
		return true;
	}
	
	public static void sourceUpdate(Subscription sub, SrcDataset src) throws Exception
	{
		
		Version version=src.getMostRecentVersion();
		//file name without gz suffix
		File targetFile=new File(DB_TMP+File.separator+sub.getFilename().substring(0, sub.getFilename().length()-3));
		FileOutputStream out=new FileOutputStream(targetFile);
		
		String prefix=version.getVersion() + "/" + sub.getFilename() + "/";
		InputStream s3in=new DSS3SequenceInputStream(prefix, version.getFsize(), client);
		
		//calculate compmd5
		MessageDigest compmd5=MessageDigest.getInstance("MD5");
		DigestInputStream compin=new DigestInputStream(s3in, compmd5);
		
		//gunzip the inputStream
		GZIPInputStream gin=new GZIPInputStream(compin);
		
		//calculate the uncompmd5
		MessageDigest uncompmd5=MessageDigest.getInstance("MD5");
		DigestInputStream uncompin=new DigestInputStream(gin, uncompmd5);
		
		IOUtils.copy(uncompin, out);
		out.close();
		uncompin.close();
		gin.close();
		compin.close();
		s3in.close();
		
		String uncompmd5Hex=new String(Hex.encodeHex(uncompmd5.digest()));
		String compmd5Hex=new String(Hex.encodeHex(compmd5.digest()));
		if (!uncompmd5Hex.equals(version.getUncompmd5()))
		{
			logger.info("\t uncompmd5 does not match. Calculated uncompmd5= "+uncompmd5Hex+" expecte uncompmd5"+version.getUncompmd5());
			throw new Exception("uncompmd5 does not match");
			
		}
		if (!compmd5Hex.equals(version.md5))
		{
			logger.info("\t compmd5 does not match. Calculated compmd5= "+compmd5Hex+" expecte compmd5"+version.getMd5());
			throw new Exception("compmd5 does not match");
		}
		
		formatDB(sub, src, version, targetFile);
		sub.setServerVersion(version.getVersion());
		sub.setModified(new Date());
		sub.setLetters(version.getLetters());
		sub.setSeqs(version.getSeqs());
		File srcFile=new File(DB_PATH+File.separator+src.getName());
		sub.setFilesize(srcFile.length());
		//sub.setFilename(targetFile.getAbsolutePath());
		DSPersister.writeSubscription(subs);
		
	}

	public static void patchUpdate(Subscription sub, SrcDataset src, Patch patch) throws Exception
	{
		
		//file name without gz suffix
		File targetFile=new File(DB_TMP+File.separator+src.getName()+".patch");
		FileOutputStream out=new FileOutputStream(targetFile);
		
		
		
		String prefix = "diff/" + patch.getDifffile()+ "/" ;
		InputStream s3in=new DSS3SequenceInputStream(prefix, patch.getDiffsize(), client);
		
		//calculate compmd5
		MessageDigest compmd5=MessageDigest.getInstance("MD5");
		DigestInputStream compin=new DigestInputStream(s3in, compmd5);
		
		//gunzip the inputStream
		GZIPInputStream gin=new GZIPInputStream(compin);
		
		
		IOUtils.copy(gin, out);
		out.close();
		gin.close();
		compin.close();
		s3in.close();
		
		String compmd5Hex=new String(Hex.encodeHex(compmd5.digest()));
		
		if (!compmd5Hex.equals(patch.getPatchmd5()))
		{
			logger.info("\t compmd5 does not match. Calculated compmd5= "+compmd5Hex+" expecte compmd5"+patch.getPatchmd5());
			throw new Exception("compmd5 does not match");
		}
		
		//formatdb
		List<String> cmds=new ArrayList<String>();
		cmds.add("fpatch");
		cmds.add(targetFile.getName());
		cmds.add(DB_PATH+File.separator+sub.getName());
		cmds.add(sub.getName());
		
		
		ProcessBuilder pb = new ProcessBuilder(cmds.toArray(new String[0]));
		pb.directory(new File(DB_TMP));
		Process process=pb.start();
		process.waitFor();
		int exitValue=process.exitValue();
		if (exitValue!=0)
		{
			InputStream errStream=process.getErrorStream();
			InputStream outStream=process.getInputStream();
			String err=IOUtils.toString(errStream);
			String out1=IOUtils.toString(outStream);
			errStream.close();
			outStream.close();
			String cmdStr=Arrays.toString(cmds.toArray(new String[0]));
			logger.warn("patch: command is "+cmdStr+". Error Stream "+err+" output is "+out1+" Exit Value is "+exitValue);
			process.destroy();
			throw new Exception("patch failed with error code "+exitValue);
		}
		process.destroy();

		File srcFile=new File(DB_TMP+File.separator+src.getName());
		
		Version version=src.getVersion(patch.getCurversion());
		formatDB(sub, src, version, srcFile);
		sub.setServerVersion(version.getVersion());
		sub.setModified(new Date());
		sub.setLetters(version.getLetters());
		sub.setSeqs(version.getSeqs());
		srcFile=new File(DB_PATH+File.separator+src.getName());
		sub.setFilesize(srcFile.length());
		//sub.setFilename(srcFile.getAbsolutePath());
		DSPersister.writeSubscription(subs);
		
	}

	
	
	public static void formatDB(Subscription sub,SrcDataset src, Version version,File srcFile) throws Exception
	{
		
		//delete interefering file
		File tmpDir=new File(DB_TMP);
		for (String junkfile: tmpDir.list(new PrefixFileFilter(srcFile.getName()+".")))
		{
			new File(junkfile).delete();
		}
		
		//formatdb
		List<String> cmds=new ArrayList<String>();
		cmds.add("btformatdb");
		cmds.add("-i");
		cmds.add(srcFile.getName());
		cmds.add("-p");
		cmds.add(sub.isProtein()?"T":"F");
		cmds.add("-l");
		cmds.add("formatdb.log");
		cmds.add("-X");
		cmds.add(String.valueOf(version.getLetters()));
		cmds.add("-t");
		cmds.add(srcFile.getName());
		if (src.isIndexable())
		{
			cmds.add("-o");
			cmds.add("T");
		}
		
		ProcessBuilder pb = new ProcessBuilder(cmds.toArray(new String[0]));
		pb.directory(new File(DB_TMP));
		Process process=pb.start();
		process.waitFor();
		int exitValue=process.exitValue();
		InputStream errStream=process.getErrorStream();
		InputStream outStream=process.getInputStream();
		String err=IOUtils.toString(errStream);
		String out=IOUtils.toString(outStream);
		errStream.close();
		outStream.close();
		String cmdStr=Arrays.toString(cmds.toArray(new String[0]));
		logger.warn("command is "+cmdStr+". Error Stream "+err+" output is "+out+" Exit Value is "+exitValue);
		
		if (exitValue!=0)
		{
			
					process.destroy();
			throw new Exception("formatdb failed with error code "+exitValue);
		} else
		{
			
		}
		process.destroy();
		
		//modify generated nal/pal file.
		File npfile=new File(DB_TMP+File.separator+srcFile.getName()+(sub.isProtein()?".pal":".nal"));
		if (npfile.exists())
		{
			Reader in=new FileReader(npfile);
			String content=IOUtils.toString(in);
			in.close();
			content=content.replaceAll(DB_TMP, DB_PATH);
			Writer out1=new FileWriter(npfile);
			out1.write(content);
			out1.close();
		}
		
		
		//delete old file
		File dbDir=new File(DB_PATH);
		for (File oldfile: tmpDir.listFiles((FileFilter)new PrefixFileFilter(srcFile.getName())))
		{
			oldfile.delete();
		}
		//move file
		for (File f: tmpDir.listFiles((FileFilter)new PrefixFileFilter(srcFile.getName())))
		{
			FileUtils.moveToDirectory(f, dbDir, false);
		}
		
	}
	
	public static boolean isDatasetValid(Subscription sub)
	{
		File srcFile = new File(DB_PATH + File.separator + sub.getName());
		if (!srcFile.exists())
		{
			logger.info("\tdataset is invalid since source file"+srcFile.getAbsolutePath()+  "does not exist");
			return false;
		}
		if (srcFile.length() != sub.getFilesize())
		{
			logger.info("\tdataset is invalid since source file size does not match");
			return false;
		}
		File dbDir = new File(DB_PATH);
		String[] formattedFiles = dbDir.list(new PrefixFileFilter(sub.getName()));
		if (formattedFiles.length < 3)
		{
			logger.info("\tdataset is invalid since less than 3 formatted segments are found");
			return false;
		}

		return true;
	}

	
	public static InputStream getDSFileFromS3(long size, String version, String s3Name) throws Exception
	{
		
		String prefix=version + "/" + s3Name + "/";
		return new DSS3SequenceInputStream(prefix, size, client);
		
	}

	

	
	public static InputStream getDiffFileFromS3(long size, String key) throws Exception
	{
		String prefix = "diff/" + key + "/" ;
		return new DSS3SequenceInputStream(prefix, size, client);
		
	}
//	public static void getSingleFileFromS3(String key, OutputStream out) throws Exception
//	{
//		
//		HttpMethod method = new GetMethod("http://bioteamds.s3.amazonaws.com/"+key);
//		//HttpMethod method = new GetMethod("http://localhost:8080/" + key);
//		method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(5, false));
//		MessageDigest md = MessageDigest.getInstance("MD5");
//
//		DigestOutputStream dout = new DigestOutputStream(out, md);
//		try
//		{
//			// Execute the method.
//			int statusCode = client.executeMethod(method);
//			if (statusCode != HttpStatus.SC_OK)
//			{
//				logger.warn("failed in downloading file " + method.getStatusLine());
//				throw new Exception(method.getStatusLine().toString());
//			}
//			String etag = method.getResponseHeader("ETag").getValue();
//
//			InputStream in = method.getResponseBodyAsStream();
//			IOUtils.copy(in, dout);
//			byte[] digestbyte = md.digest();
//			String hex = new String(Hex.encodeHex(digestbyte));
//			if (etag.startsWith("\""))
//				hex = '"' + hex + '"';
//			if (!etag.equals(hex))
//			{
//				throw new Exception("md5 does not match");
//			}
//		} finally
//		{
//			// Release the connection.
//			method.releaseConnection();
//		}
//
//	}

}
